#!/usr/bin/env python
#-*-encoding:utf-8-*-
'''
Created on 2015年2月3日

@author: chenyongbing
'''
import sys,os
current_dir = os.path.dirname(__file__)
import pexpect
import commands
import time,datetime
import logging

class MergerRawdataFromRemote(object):
    '''
    classdocs
    '''


    def __init__(self):
        '''
        Constructor
        '''
        self.hosts = ['172.20.1.40',
                      '172.20.1.84',
                      '172.20.1.85',
                      '172.20.1.86',
                      '172.20.1.87',
                      '172.20.1.92',
                      '172.20.1.93']
        self.user = 'root'
        self.passwd = 'Usr12345'
        
        #self.init_logger()
        
    def init_logger(self):
        LOG_FILE = 'tst.log'
        
        handler = logging.handlers.RotatingFileHandler(LOG_FILE, maxBytes = 1024*1024*10, backupCount = 5) # 实例化handler 
        fmt = '%(asctime)s - %(filename)s:%(lineno)s - %(name)s - %(message)s'
        
        formatter = logging.Formatter(fmt)   # 实例化formatter
        handler.setFormatter(formatter)      # 为handler添加formatter
        
        logging = logging.getLogger('tst')    # 获取名为tst的logger
        logging.addHandler(handler)           # 为logger添加handler
        logging.setLevel(logging.DEBUG)
        
    def get_slist_between_st_et(self,st,et):
        u'''获取2个时间间的所有日期'''
        dlist = []
        count = 0
        nday = (datetime.date(int(st[:4]),int(st[5:7]),int(st[8:]))+ datetime.timedelta(-1)).strftime('%Y-%m-%d')
        while nday < et:
            nday = (datetime.date(int(st[:4]),int(st[5:7]),int(st[8:]))+ datetime.timedelta(count)).strftime('%Y-%m-%d')
            count+=1
            dlist.append(nday)
        return dlist
    def scp_from_remote(self,ip, user, passwd, dst_path, filename,timeout=30):
        if os.path.isdir(filename):
            cmdline = 'scp -r %s@%s:%s %s' % (user, ip,filename, dst_path)
        else:
            cmdline = 'scp  %s@%s:%s %s' % ( user, ip,filename, dst_path)
        #print cmdline
        try:
            t1 = time.time()
            child = pexpect.spawn(cmdline,timeout=timeout)
            child.expect('password:')
            child.sendline(passwd)
            child.expect(pexpect.EOF)
            t2 = time.time()
     
        #child.interact()
        #child.read()
        #child.expect('$')
            logging.info( "use %s sec to download file %s:%s"%(t2-t1,ip,filename))
            return True
        except Exception , err:
            try:
                errMsg = err.message
                #print errMsg
            except:
                errMsg = None
            if errMsg==None:
                return True
            if re.search('End Of File',errMsg):
                return True
            if re.search('No such file or directory',errMsg):
                logging.error('download %s:%s not exists'%(ip,filename))
                return True
            logging.error( "download %s:%s faild"%(ip,filename))   
            return False
    
    def merger_rawdata(self,day=None,remotePath='/usr/java/data/rawdata/netty/',localPath='/data/rawdatatmp/',rawPath='/usr/java/data/rawdata/',timeout=30):
        if day==None:
            day = (datetime.date.today()-datetime.timedelta(1)).strftime('%Y-%m-%d')
        yesterday = (datetime.date.today()-datetime.timedelta(1)).strftime('%Y-%m-%d')
        rawfile = rawPath+'rawdata.%s.txt'%day
        rawtarfile = rawfile+'.tar.gz'
        if not os.path.exists(rawtarfile):
            for host in self.hosts:
                logging.info( "start scp rawdata from %s"%(host))
                try:
                    commands.getstatusoutput('mkdir -p %s/%s'%(localPath,host))
                except:pass
                if os.path.exists(rawtarfile):
                    break
                if day != yesterday:
                    remoteFiles = remotePath+'rawdata.*%s.*'%day
                else:
                    remoteFiles = remotePath+'rawdata.*%s.*.txt'%day
    #             while 1 :
    #                 if self.scp_from_remote(host, self.user, self.passwd, localPath+host+'/', remoteFiles, timeout):
    #                     break
                while 1:
                    cmd = 'scp %s:%s %s'%(host,remoteFiles,localPath+host+'/')
                    
                    out = commands.getstatusoutput(cmd)
                    #print out
                    if int(out[0])==0 or re.search('No such file or directory',out[1]):break
                    
        while 1:
            if  os.path.exists(rawfile) and os.path.exists(rawtarfile):
                break
            localFiles = localPath+'/*/rawdata.*%s.*.txt'%day
            for hostPath in os.listdir(localPath):
                for tfile in os.listdir(localPath+'/'+hostPath):
                    if re.search('.tar.gz',tfile):
                        os.chdir(localPath+'/'+hostPath)
                        commands.getstatusoutput('tar -xzvf %s'%(localPath+'/'+hostPath+'/'+tfile))
            commands.getstatusoutput('cat %s > %s'%(localFiles,rawfile))
        while 1:
            os.chdir(rawPath)
            if os.path.exists(rawfile):
                tarcmd = 'tar -czvf %s.tar.gz %s'%(rawfile,'rawdata.%s.txt'%day)
                out = commands.getstatusoutput(tarcmd)
                #print out
                if int(out[0])==0:
                    break
        commands.getstatusoutput('rm -rf %s/*/*'%(localPath)) 
            
    
if __name__ == '__main__':
    import argparse,re,datetime
    parser = argparse.ArgumentParser(description='args')
    
    parser.add_argument('--startTime',metavar=u"",default=None)
    parser.add_argument('--endTime',metavar=u"requests",default=None)
    parser.add_argument('--remotePath',metavar=u"remote path",default='/usr/java/data/rawdata/netty/')
    parser.add_argument('--localPath',metavar=u"local path",default='/data/rawdatatmp/')
    parser.add_argument('--rawPath',metavar=u"raw path",default='/usr/java/data/rawdata/')
    args = parser.parse_args()
    startTime = args.startTime
    endTime = args.endTime
    remotePath = args.remotePath
    localPath = args.localPath
    rawPath = args.rawPath
    logging.basicConfig(level=logging.DEBUG)
    if startTime==None:
        startTime = (datetime.date.today()-datetime.timedelta(1)).strftime('%Y-%m-%d')
    if endTime == None:
        endTime = (datetime.date.today()-datetime.timedelta(1)).strftime('%Y-%m-%d')
    
    myMergerRawdataFromRemote = MergerRawdataFromRemote()
    days = myMergerRawdataFromRemote.get_slist_between_st_et(startTime, endTime)
    for day in days:
        myMergerRawdataFromRemote.merger_rawdata(day=day,remotePath=remotePath,localPath=localPath,rawPath=rawPath)
